{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy.io as sio\n",
    "import scipy.sparse as ss\n",
    "import _pickle as cPickle\n",
    "import os\n",
    "import scipy.spatial.distance as ssd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 用户和item索引\n",
    "users_index = cPickle.load(open('ml-100k/users_index.pkl','rb'))\n",
    "items_index = cPickle.load(open('ml-100k/items_index.pkl','rb'))\n",
    "\n",
    "n_users = len(users_index)\n",
    "n_items = len(items_index)\n",
    "\n",
    "# 到排序\n",
    "#每个用户打过分的电影\n",
    "user_items = cPickle.load(open('ml-100k/user_items.pkl','rb'))\n",
    "item_users = cPickle.load(open('ml-100k/item_users.pkl','rb'))\n",
    "\n",
    "# 用户关系矩阵\n",
    "user_item_scores = sio.mmread('ml-100k/user_item_scores')\n",
    "user_item_scores = user_item_scores.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 计算每个用户的平均打分\n",
    "user_ms = np.zeros(n_users)\n",
    "\n",
    "for i in range(n_users):\n",
    "    score = 0;\n",
    "    n_item = 0;\n",
    "    for item in user_items[i]:\n",
    "        score += user_item_scores[i,item]\n",
    "        n_item += 1\n",
    "    if n_item > 0:\n",
    "        user_ms[i] = score/n_item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def item_similarity(iid1,iid2):\n",
    "    su = {}\n",
    "    for user in item_users[iid1]: # 找出item1打分的用户\n",
    "        if user in item_users[iid2]:  # 如果item1打分的用户也再item2中则为有效的有效的用户\n",
    "            su[user] = 1\n",
    "    n = len(su)\n",
    "    if n == 0:\n",
    "        similarity = 0.0\n",
    "        return similarity\n",
    "    # 对item1的有效打分\n",
    "    s1 = np.array([user_item_scores[user,iid1]-user_ms[user] for user in su])\n",
    "    # 对item2的有效打分\n",
    "    s2 = np.array([user_item_scores[user,iid2]-user_ms[user] for user in su])\n",
    "    \n",
    "    similarity = 1 - ssd.cosine(s1,s2)\n",
    "    \n",
    "    if np.isnan(similarity):\n",
    "        similarity = 0.0\n",
    "    return similarity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4276177764039042\n"
     ]
    }
   ],
   "source": [
    "iid1 = 8\n",
    "iid2 = 34\n",
    "print(item_similarity(iid1,iid2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ii = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jay/.local/lib/python3.7/site-packages/scipy/spatial/distance.py:720: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  dist = 1.0 - uv / np.sqrt(uu * vv)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ii = 100\n",
      "ii = 200\n",
      "ii = 300\n",
      "ii = 400\n",
      "ii = 500\n",
      "ii = 600\n",
      "ii = 700\n",
      "ii = 800\n",
      "ii = 900\n",
      "ii = 1000\n",
      "ii = 1100\n",
      "ii = 1200\n",
      "ii = 1300\n",
      "ii = 1400\n",
      "ii = 1500\n",
      "ii = 1600\n"
     ]
    }
   ],
   "source": [
    "# 预算好所有物品之间的相似度，可以加快后边的计算\n",
    "# 对于物品较少，物品比较固定的系统实用\n",
    "item_similarity_matrix = np.matrix(np.zeros(shape = (n_items,n_items)),float)\n",
    "\n",
    "for ii in range(n_items):\n",
    "    item_similarity_matrix[ii,ii] = 1.0\n",
    "    if ii % 100 == 0:\n",
    "        print('ii = {}'.format(ii))\n",
    "    for ij in range(ii+1,n_items):\n",
    "        item_similarity_matrix[ij,ii]= item_similarity(ii,ij)\n",
    "        item_similarity_matrix[ii,ij] = item_similarity_matrix[ij,ii]\n",
    "cPickle.dump(item_similarity_matrix,open('ml-100k/item_similarity_matrix.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 预测用户对item的打分\n",
    "# 利用用户打过分的item计算相似度\n",
    "def Item_CF_pred(uid,iid):\n",
    "    sim_accumnlate = 0.0\n",
    "    rat_acc = 0.0\n",
    "    for item_id in user_items[uid]: # 这个用户对哪些物品打过分\n",
    "        # 获取这个物品之间的相识度\n",
    "        sim = item_similarity_matrix[item_id,iid]\n",
    "        if sim != 0:\n",
    "            rat_acc += sim * user_item_scores[uid,item_id]\n",
    "            sim_accumnlate += np.abs(sim)\n",
    "    # 估计最后结果，如果为0就给均分\n",
    "    if sim_accumnlate != 0:\n",
    "        score = rat_acc/sim_accumnlate\n",
    "    else:\n",
    "        score = user_ms[uid]\n",
    "    \n",
    "    if score < 0:\n",
    "        score = 0.0\n",
    "    return score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4446581134818482\n"
     ]
    }
   ],
   "source": [
    "print(Item_CF_pred(3,5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.11182986 -0.0621802  -0.23122191 ...  0.          1.\n",
      "  -1.        ]]\n"
     ]
    }
   ],
   "source": [
    "print(np.array(item_similarity_matrix[5,:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 利用用户打过分的item中，与item相似的n_Knns个物品计算预测打分\n",
    "# PR和覆盖度性能最好\n",
    "def Item_CF_pred1(uid,iid,n_Knns):\n",
    "    sim_accumnlate = 0.0\n",
    "    rat_acc = 0.0\n",
    "    n_nn_items = 0 #用于计算和n_Knns判断\n",
    "    cur_items_similarity = np.array(item_similarity_matrix[iid,:])\n",
    "    cur_items_similarity = cur_items_similarity.flatten() # 返回折叠成一维数组的副本\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(cur_items_similarity))),reverse=True) # 根据物品相似度排序（相似度，物品索引）\n",
    "    for i in range(len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        if n_nn_items > n_Knns:\n",
    "            break\n",
    "        if cur_item_index in user_items[uid]: # 再当前用户打过分的商品里边\n",
    "            sim = item_similarity_matrix[iid, cur_item_index]\n",
    "            if sim != 0:\n",
    "                rat_acc += sim * user_item_scores[uid,cur_item_index]\n",
    "                sim_accumnlate += np.abs(sim)\n",
    "            n_nn_items +=1\n",
    "    \n",
    "    if sim_accumnlate != 0:\n",
    "        score = rat_acc/sim_accumnlate\n",
    "    else:\n",
    "        score = user_ms[uid]\n",
    "    \n",
    "    if score < 0:\n",
    "        score = 0.0\n",
    "    return score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35828119340528614\n"
     ]
    }
   ],
   "source": [
    "print(Item_CF_pred1(3,5,6)) # 和自己打分里边相似度高的前几个一样？直接找到自己打过分的然后和新来的看下相似度，然后排序不就可以了？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 只截取前n_Knns个，然后从用户打过分的里边看有没有打过分的\n",
    "def Item_CF_pred2(uid,iid,n_Knns):\n",
    "    sim_accumnlate = 0.0\n",
    "    rat_acc = 0.0\n",
    "    cur_items_similarity = np.array(item_similarity_matrix[iid,:])\n",
    "    cur_items_similarity = cur_items_similarity.flatten() # 返回折叠成一维数组的副本\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(cur_items_similarity))),reverse=True)[: n_Knns] # 根据物品相似度排序（相似度，物品索引）\n",
    "    for i in range(len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        if cur_item_index in user_items[uid]: # 再当前用户打过分的商品里边\n",
    "            sim = item_similarity_matrix[iid, cur_item_index]\n",
    "            if sim != 0:\n",
    "                rat_acc += sim * user_item_scores[uid,cur_item_index]\n",
    "                sim_accumnlate += np.abs(sim)\n",
    "    \n",
    "    if sim_accumnlate != 0:\n",
    "        score = rat_acc/sim_accumnlate\n",
    "    else:\n",
    "        score = user_ms[uid]\n",
    "    \n",
    "    if score < 0:\n",
    "        score = 0.0\n",
    "    return score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.35828119340528614\n"
     ]
    }
   ],
   "source": [
    "print(Item_CF_pred1(3,5,6)) # 最高相似的都在自己列表里边？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对物品进行推荐\n",
    "\n",
    "def recommend(user):\n",
    "    cur_user_id = users_index[user]\n",
    "    \n",
    "    # 训练集中用户打过分的item\n",
    "    cur_user_items = user_items[cur_user_id]\n",
    "    \n",
    "    # 该用户对所有物品打分\n",
    "    user_item_scores = np.zeros(n_items)\n",
    "    \n",
    "    # 预测打分\n",
    "    for i in range(n_items):\n",
    "        if i not in cur_user_items:\n",
    "            user_item_scores[i] = Item_CF_pred1(cur_user_id,i,5)\n",
    "    # 用元组来存（分数，物品id）\n",
    "    sort_index = sorted(((e,i) for i,e in enumerate(list(user_item_scores))),reverse=True)\n",
    "    columns = ['item_id','score']\n",
    "    df = pd.DataFrame(columns=columns)\n",
    "    \n",
    "    for i in range(0,len(sort_index)):\n",
    "        cur_item_index = sort_index[i][1]\n",
    "        # 把index转化乘list然后通过index定位value所在位置，然后再将key（物品真正的id）转化成list，找到真正的item id\n",
    "        cur_item = list(items_index.keys())[list(items_index.values()).index(cur_item_index)]\n",
    "        \n",
    "        if ~np.isnan(sort_index[i][0]) and cur_item_index not in cur_user_items:\n",
    "            df.loc[len(df)] = [cur_item,sort_index[i][0]]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>887431973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>875693118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>12</td>\n",
       "      <td>5</td>\n",
       "      <td>878542960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>874965706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>875073198</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  item_id  rating  timestamp\n",
       "0        1        6       5  887431973\n",
       "1        1       10       3  875693118\n",
       "2        1       12       5  878542960\n",
       "3        1       14       5  874965706\n",
       "4        1       17       3  875073198"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "triplet_cols = ['user_id','item_id','rating','timestamp']\n",
    "df_triplet_test = pd.read_csv('ml-100k/u1.test',sep='\\t',encoding='Latin-1',names=triplet_cols)\n",
    "df_triplet_test.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "599 is new item. \n",
      "711 is new item. \n",
      "814 is new item. \n",
      "830 is new item. \n",
      "852 is new item. \n",
      "857 is new item. \n",
      "1156 is new item. \n",
      "1236 is new item. \n",
      "1309 is new item. \n",
      "1310 is new item. \n",
      "1320 is new item. \n",
      "1343 is new item. \n",
      "1348 is new item. \n",
      "1364 is new item. \n",
      "1373 is new item. \n",
      "1457 is new item. \n",
      "1458 is new item. \n",
      "1492 is new item. \n",
      "1493 is new item. \n",
      "1498 is new item. \n",
      "1505 is new item. \n",
      "1520 is new item. \n",
      "1533 is new item. \n",
      "1536 is new item. \n",
      "1543 is new item. \n",
      "1557 is new item. \n",
      "1561 is new item. \n",
      "1562 is new item. \n",
      "1563 is new item. \n",
      "1565 is new item. \n",
      "1582 is new item. \n",
      "1586 is new item. \n"
     ]
    }
   ],
   "source": [
    "# 统计总的用户\n",
    "unique_users_test = df_triplet_test['user_id'].unique()\n",
    "\n",
    "# 为每个用户推荐20个商品\n",
    "n_rec_items = 20\n",
    "\n",
    "#性能评价计算精确率和召回率\n",
    "n_hits = 0\n",
    "n_total_rec_items = 0\n",
    "n_test_items = 0\n",
    "\n",
    "#所有被推荐商品的集合，用于计算覆盖度\n",
    "all_rec_items = set()\n",
    "\n",
    "# 残差平方和，用于计算RMSE\n",
    "rss_test = 0.0\n",
    "\n",
    "# 对每个测试用户\n",
    "for user in unique_users_test:\n",
    "    if user not in users_index:\n",
    "        print('{} is new user'.format(user))\n",
    "        continue\n",
    "    user_records_test = df_triplet_test[df_triplet_test.user_id == user]\n",
    "    \n",
    "    rec_items = recommend(user)\n",
    "    \n",
    "    for i in range(n_rec_items):\n",
    "        item = rec_items.iloc[i]['item_id']\n",
    "        \n",
    "        if item in user_records_test['item_id'].values:\n",
    "            n_hits +=1\n",
    "            \n",
    "        all_rec_items.add(item)\n",
    "    \n",
    "    # 计算rmse\n",
    "    for i in range(user_records_test.shape[0]):\n",
    "        item = user_records_test.iloc[i]['item_id']\n",
    "        score = user_records_test.iloc[i]['rating']\n",
    "        \n",
    "        df1 = rec_items[rec_items.item_id == item]\n",
    "        if df1.shape[0] == 0:\n",
    "            print('{} is new item. '.format(item))\n",
    "            continue\n",
    "        pre_score = df1['score'].values[0]\n",
    "        rss_test += (pre_score - score)**2\n",
    "    # 推荐item 总数\n",
    "    n_total_rec_items += n_rec_items\n",
    "    n_test_items += user_records_test.shape[0]\n",
    "\n",
    "precision = n_hits / (1.0 * n_total_rec_items)\n",
    "recall = n_hits / (1.0 * n_test_items)\n",
    "\n",
    "# 覆盖率\n",
    "coverage = len(all_rec_items) / (1.0 * n_items)\n",
    "\n",
    "# 打分均方误差\n",
    "rmse = np.sqrt(rss_test/df_triplet_test.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0434640522875817"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "precision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.01995"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5642424242424242"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coverage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3.5421199531458316"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rmse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
